# -*- coding: utf8 -*-
import urllib2
import urllib
import time
import re
import os
import time
import CatchXml
import down
import download
import sqlite3testdown
from threading import Thread
from Queue import Queue
import threading
mylock = threading.RLock()
mylock2 = threading.RLock()
Breakpoint=0
sitelist=[]#存放http://fenlei.baike.com下类别词条数
thlist=[]
queue=Queue()
queue_temp=Queue()
def run(sites):#线程run方法 调用down()方法实现下载
    global Breakpoint
    while True:
        if sites.empty():
            print('finish!')
            break
        category=sites.get()
        try:
            #print('sites size')
            #print(sites.qsize())
            sqlite3testdown.down(category)
            queue_temp.get()
            print('1')
        except:
            time.sleep(0.1)
            sites.put(category)
        mylock.acquire()
        p=367-queue_temp.qsize()#367=len(sitelist)
        print(str(p)+'x')
        file = open('E:/point.txt','w')
        file.write(str(p))
        file.close()
        mylock.release()
url=('http://fenlei.baike.com/')#起始网页
sitelist=CatchXml.add_func(url)
print len(sitelist)
file = open('E:/point.txt','r')#从文件中读出断点信息
Strpoint=file.readline()
file.close()
Breakpoint=int(Strpoint)
if Breakpoint==len(sitelist):#如果已经下载完成 退出程序
    print('no Thread will start!')
    exit()
for x in sitelist:#将所有词条类别压入队列
    queue.put(x)
    queue_temp.put(x)
for i in range(Breakpoint):#从队列中去除断点信息数的条数
    queue.get()
    queue_temp.get()
for i in range(0,200):
    thlist.append(threading.Thread(target=run,args=(queue,)))
for i in thlist:
    i.start()
